{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Simple tutorial for using TensorFlow to compute polynomial regression.\n",
    "\n",
    "Parag K. Mital, Jan. 2016\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %% Imports\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %% Let's create some toy data\n",
    "plt.ion()\n",
    "n_observations = 100\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "xs = np.linspace(-3, 3, n_observations)\n",
    "ys = np.sin(xs) + np.random.uniform(-0.5, 0.5, n_observations)\n",
    "ax.scatter(xs, ys)\n",
    "fig.show()\n",
    "plt.draw()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %% tf.placeholders for the input and output of the network. Placeholders are\n",
    "# variables which we need to fill in when we are ready to compute the graph.\n",
    "X = tf.placeholder(tf.float32)\n",
    "Y = tf.placeholder(tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %% Instead of a single factor and a bias, we'll create a polynomial function\n",
    "# of different polynomial degrees.  We will then learn the influence that each\n",
    "# degree of the input (X^0, X^1, X^2, ...) has on the final output (Y).\n",
    "Y_pred = tf.Variable(tf.random_normal([1]), name='bias')\n",
    "for pow_i in range(1, 5):\n",
    "    W = tf.Variable(tf.random_normal([1]), name='weight_%d' % pow_i)\n",
    "    Y_pred = tf.add(tf.mul(tf.pow(X, pow_i), W), Y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %% Loss function will measure the distance between our observations\n",
    "# and predictions and average over them.\n",
    "cost = tf.reduce_sum(tf.pow(Y_pred - Y, 2)) / (n_observations - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %% if we wanted to add regularization, we could add other terms to the cost,\n",
    "# e.g. ridge regression has a parameter controlling the amount of shrinkage\n",
    "# over the norm of activations. the larger the shrinkage, the more robust\n",
    "# to collinearity.\n",
    "# cost = tf.add(cost, tf.mul(1e-6, tf.global_norm([W])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %% Use gradient descent to optimize W,b\n",
    "# Performs a single step in the negative gradient\n",
    "learning_rate = 0.01\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %% We create a session to use the graph\n",
    "n_epochs = 1000\n",
    "with tf.Session() as sess:\n",
    "    # Here we tell tensorflow that we want to initialize all\n",
    "    # the variables in the graph so we can use them\n",
    "    sess.run(tf.initialize_all_variables())\n",
    "\n",
    "    # Fit all training data\n",
    "    prev_training_cost = 0.0\n",
    "    for epoch_i in range(n_epochs):\n",
    "        for (x, y) in zip(xs, ys):\n",
    "            sess.run(optimizer, feed_dict={X: x, Y: y})\n",
    "\n",
    "        training_cost = sess.run(\n",
    "            cost, feed_dict={X: xs, Y: ys})\n",
    "        print(training_cost)\n",
    "\n",
    "        if epoch_i % 100 == 0:\n",
    "            ax.plot(xs, Y_pred.eval(\n",
    "                feed_dict={X: xs}, session=sess),\n",
    "                    'k', alpha=epoch_i / n_epochs)\n",
    "            fig.show()\n",
    "            plt.draw()\n",
    "\n",
    "        # Allow the training to quit if we've reached a minimum\n",
    "        if np.abs(prev_training_cost - training_cost) < 0.000001:\n",
    "            break\n",
    "        prev_training_cost = training_cost\n",
    "ax.set_ylim([-3, 3])\n",
    "fig.show()\n",
    "plt.waitforbuttonpress()"
   ]
  }
 ],
 "metadata": {
  "language": "python"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
